package web.newcrawler;

import com.google.gson.Gson;
import org.apache.struts2.ServletActionContext;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import javax.servlet.http.HttpSession;
import java.util.ArrayList;
import java.util.Set;

public class AnalyProcessor implements PageProcessor {
    //获取Session对象
    private HttpSession session = ServletActionContext.getRequest().getSession();

    @Override
    public void process(Page page) {

        System.out.println("页面地址是:" + page.getUrl().toString());
//        System.out.println("页面内容如下:");
//        System.out.println(page.getHtml());

        //Jsoup解析html页面信
        Document html = Jsoup.parse(page.getHtml().toString());

        //获取所有表格
        Elements tables = html.getElementsByTag("table");
        //看看是否有表格存在
        if (!tables.isEmpty()){

            ArrayList<String> tableList = new ArrayList<>();

            for (Element table : tables) {
                tableList.add(table.toString());
            }

            session.setAttribute("table",tableList);
        }

        //获取所有文字
        Elements paragraphs = html.getElementsByTag("p");
        //看看是否有文字存在
        if (!paragraphs.isEmpty()){

            ArrayList<String> paragraphList = new ArrayList<>();

            for (Element paragraph : paragraphs) {
                paragraphList.add(paragraph.toString());
            }

            session.setAttribute("paragraph",paragraphList);
        }

    }

    private Site site = Site.me();

    @Override
    public Site getSite() {

        //获取到cookies
        Set<Cookie> cookies = (Set<Cookie>) session.getAttribute("cookies");
        //遍历cookie集合
        if (!cookies.isEmpty()){
            for (Cookie cookie : cookies) {
                site.addCookie(cookie.getName(), cookie.getValue());
            }
        }

        //site的基本配置
        site.setTimeOut(10 * 1000)    //设置超时时间
                .setRetrySleepTime(3000)    //设置重试间隔时间
                .setCharset("utf-8")
                .setRetryTimes(3)  //设置重试的次数
                .setDisableCookieManagement(true)
                ;
        return site;
    }
}
